# -*- coding: utf-8 -*-

import re
import morfologik
import stemmer

class Tokenizer():
    def __init__(self, file, word_processor=None):
        '''param indicates, what should be done with a token: lemmatization (l), stemming (s) or nothing (n)'''
        self.__file = file
        self.__word_processor = word_processor

    def __process_line(self, line) :
        l = re.sub(r'[^a-zA-Z0-9 ]', '', line)
        return l.lower()
    
    def next(self):
        i = 0
        for line in self.__file:
            l = self.__process_line(line)
            for token in l.split(" "):
                i = i + 1
                if token:
                    if self.__word_processor == None :
                        yield (i, token)
                    else :
                        yield (i, self.__word_processor.process(token) )
        
    def next_touple(self):
        for line in self.__file:
            l = process_line(line)
            tokens = l.split(" ")
            first = None
            while not first and tokens:
                first  = tokens[0]
                tokens = tokens[1:]

            for token in tokens:
                if token:
                    yield (first, token)
                    first = token


if __name__ == "__main__":
#    m = morfologik.Morfologik()
    s = stemmer.Stemmer()
    tokens = Tokenizer(open("example.txt"), word_processor=s)
    for token in tokens.next():
        print token
